includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         /** @var CF_Connection */
  49         protected $conn; // Swift connection handle
  50         protected $sessionStarted = 0; // integer UNIX timestamp
  51
  52         /** @var CloudFilesException */
  53         protected $connException;
  54         protected $connErrorTime = 0; // UNIX timestamp
  55
  56         /** @var BagOStuff */
  57         protected $srvCache;
  58
  59         /** @var ProcessCacheLRU */
  60         protected $connContainerCache; // container object cache
  61
  62         /**
  63          * @see FileBackendStore::__construct()
  64          * Additional $config params include:
  65          *   - swiftAuthUrl       : Swift authentication server URL
  66          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  67          *   - swiftKey           : Swift authentication key for the above user
  68          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  69          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  70          *                          If set, then views of public containers are assumed to go
  71          *                          through this user. If not set, then public containers are
  72          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  73          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  74          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  75          *                          If files may likely change, this should probably not exceed
  76          *                          a few days. For example, deletions may take this long to apply.
  77          *                          If object purging is enabled, however, this is not an issue.
  78          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  79          *   - shardViaHashLevels : Map of container names to sharding config with:
  80          *                             - base   : base of hash characters, 16 or 36
  81          *                             - levels : the number of hash levels (and digits)
  82          *                             - repeat : hash subdirectories are prefixed with all the
  83          *                                        parent hash directory names (e.g. "a/ab/abc")
  84          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC, XCache, ect.
  85          *                          If those are not available, then the main cache will be used.
  86          *                          This is probably insecure in shared hosting environments.
  87          */
  88         public function __construct( array $config ) {
  89                 parent::__construct( $config );
  90                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  91                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  92                 }
  93                 // Required settings
  94                 $this->auth = new CF_Authentication(
  95                         $config['swiftUser'],
  96                         $config['swiftKey'],
  97                         null, // account; unused
  98                         $config['swiftAuthUrl']
  99                 );
 100                 // Optional settings
 101                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 102                         ? $config['swiftAuthTTL']
 103                         : 5 * 60; // some sane number
 104                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 105                         ? $config['swiftAnonUser']
 106                         : '';
 107                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 108                         ? $config['shardViaHashLevels']
 109                         : '';
 110                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 111                         ? $config['swiftUseCDN']
 112                         : false;
 113                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 114                         ? $config['swiftCDNExpiry']
 115                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 116                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 117                         ? $config['swiftCDNPurgable']
 118                         : true;
 119                 // Cache container information to mask latency
 120                 $this->memCache = wfGetMainCache();
 121                 // Process cache for container info
 122                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 123                 // Cache auth token information to avoid RTTs
 124                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 125                         if ( php_sapi_name() === 'cli' ) {
 126                                 $this->srvCache = wfGetMainCache(); // preferrably memcached
 127                         } else {
 128                                 try { // look for APC, XCache, WinCache, ect...
 129                                         $this->srvCache = ObjectCache::newAccelerator( array() );
 130                                 } catch ( Exception $e ) {}
 131                         }
 132                 }
 133                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 134         }
 135
 136         /**
 137          * @see FileBackendStore::resolveContainerPath()
 138          * @return null
 139          */
 140         protected function resolveContainerPath( $container, $relStoragePath ) {
 141                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 142                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 143                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 144                         return null; // too long for Swift
 145                 }
 146                 return $relStoragePath;
 147         }
 148
 149         /**
 150          * @see FileBackendStore::isPathUsableInternal()
 151          * @return bool
 152          */
 153         public function isPathUsableInternal( $storagePath ) {
 154                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 155                 if ( $rel === null ) {
 156                         return false; // invalid
 157                 }
 158
 159                 try {
 160                         $this->getContainer( $container );
 161                         return true; // container exists
 162                 } catch ( NoSuchContainerException $e ) {
 163                 } catch ( CloudFilesException $e ) { // some other exception?
 164                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 165                 }
 166
 167                 return false;
 168         }
 169
 170         /**
 171          * @param $disposition string Content-Disposition header value
 172          * @return string Truncated Content-Disposition header value to meet Swift limits
 173          */
 174         protected function truncDisp( $disposition ) {
 175                 $res = '';
 176                 foreach ( explode( ';', $disposition ) as $part ) {
 177                         $part = trim( $part );
 178                         $new  = ( $res === '' ) ? $part : "{$res};{$part}";
 179                         if ( strlen( $new ) <= 255 ) {
 180                                 $res = $new;
 181                         } else {
 182                                 break; // too long; sigh
 183                         }
 184                 }
 185                 return $res;
 186         }
 187
 188         /**
 189          * @see FileBackendStore::doCreateInternal()
 190          * @return Status
 191          */
 192         protected function doCreateInternal( array $params ) {
 193                 $status = Status::newGood();
 194
 195                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 196                 if ( $dstRel === null ) {
 197                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 198                         return $status;
 199                 }
 200
 201                 // (a) Check the destination container and object
 202                 try {
 203                         $dContObj = $this->getContainer( $dstCont );
 204                         if ( empty( $params['overwrite'] ) &&
 205                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 206                         {
 207                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 208                                 return $status;
 209                         }
 210                 } catch ( NoSuchContainerException $e ) {
 211                         $status->fatal( 'backend-fail-create', $params['dst'] );
 212                         return $status;
 213                 } catch ( CloudFilesException $e ) { // some other exception?
 214                         $this->handleException( $e, $status, __METHOD__, $params );
 215                         return $status;
 216                 }
 217
 218                 // (b) Get a SHA-1 hash of the object
 219                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 220
 221                 // (c) Actually create the object
 222                 try {
 223                         // Create a fresh CF_Object with no fields preloaded.
 224                         // We don't want to preserve headers, metadata, and such.
 225                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 226                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 227                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 228                         // The MD5 here will be checked within Swift against its own MD5.
 229                         $obj->set_etag( md5( $params['content'] ) );
 230                         // Use the same content type as StreamFile for security
 231                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 232                         if ( !strlen( $obj->content_type ) ) { // special case
 233                                 $obj->content_type = 'unknown/unknown';
 234                         }
 235                         // Set the Content-Disposition header if requested
 236                         if ( isset( $params['disposition'] ) ) {
 237                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 238                         }
 239                         if ( !empty( $params['async'] ) ) { // deferred
 240                                 $op = $obj->write_async( $params['content'] );
 241                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
 242                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 243                                         $status->value->affectedObjects[] = $obj;
 244                                 }
 245                         } else { // actually write the object in Swift
 246                                 $obj->write( $params['content'] );
 247                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 248                                         $this->purgeCDNCache( array( $obj ) );
 249                                 }
 250                         }
 251                 } catch ( CDNNotEnabledException $e ) {
 252                         // CDN not enabled; nothing to see here
 253                 } catch ( BadContentTypeException $e ) {
 254                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 255                 } catch ( CloudFilesException $e ) { // some other exception?
 256                         $this->handleException( $e, $status, __METHOD__, $params );
 257                 }
 258
 259                 return $status;
 260         }
 261
 262         /**
 263          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 264          */
 265         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 266                 try {
 267                         $cfOp->getLastResponse();
 268                 } catch ( BadContentTypeException $e ) {
 269                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 270                 }
 271         }
 272
 273         /**
 274          * @see FileBackendStore::doStoreInternal()
 275          * @return Status
 276          */
 277         protected function doStoreInternal( array $params ) {
 278                 $status = Status::newGood();
 279
 280                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 281                 if ( $dstRel === null ) {
 282                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 283                         return $status;
 284                 }
 285
 286                 // (a) Check the destination container and object
 287                 try {
 288                         $dContObj = $this->getContainer( $dstCont );
 289                         if ( empty( $params['overwrite'] ) &&
 290                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 291                         {
 292                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 293                                 return $status;
 294                         }
 295                 } catch ( NoSuchContainerException $e ) {
 296                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 297                         return $status;
 298                 } catch ( CloudFilesException $e ) { // some other exception?
 299                         $this->handleException( $e, $status, __METHOD__, $params );
 300                         return $status;
 301                 }
 302
 303                 // (b) Get a SHA-1 hash of the object
 304                 $sha1Hash = sha1_file( $params['src'] );
 305                 if ( $sha1Hash === false ) { // source doesn't exist?
 306                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 307                         return $status;
 308                 }
 309                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 310
 311                 // (c) Actually store the object
 312                 try {
 313                         // Create a fresh CF_Object with no fields preloaded.
 314                         // We don't want to preserve headers, metadata, and such.
 315                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 316                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 317                         // The MD5 here will be checked within Swift against its own MD5.
 318                         $obj->set_etag( md5_file( $params['src'] ) );
 319                         // Use the same content type as StreamFile for security
 320                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 321                         if ( !strlen( $obj->content_type ) ) { // special case
 322                                 $obj->content_type = 'unknown/unknown';
 323                         }
 324                         // Set the Content-Disposition header if requested
 325                         if ( isset( $params['disposition'] ) ) {
 326                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 327                         }
 328                         if ( !empty( $params['async'] ) ) { // deferred
 329                                 wfSuppressWarnings();
 330                                 $fp = fopen( $params['src'], 'rb' );
 331                                 wfRestoreWarnings();
 332                                 if ( !$fp ) {
 333                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 334                                 } else {
 335                                         $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
 336                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
 337                                         $status->value->resourcesToClose[] = $fp;
 338                                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 339                                                 $status->value->affectedObjects[] = $obj;
 340                                         }
 341                                 }
 342                         } else { // actually write the object in Swift
 343                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 344                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 345                                         $this->purgeCDNCache( array( $obj ) );
 346                                 }
 347                         }
 348                 } catch ( CDNNotEnabledException $e ) {
 349                         // CDN not enabled; nothing to see here
 350                 } catch ( BadContentTypeException $e ) {
 351                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 352                 } catch ( IOException $e ) {
 353                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 354                 } catch ( CloudFilesException $e ) { // some other exception?
 355                         $this->handleException( $e, $status, __METHOD__, $params );
 356                 }
 357
 358                 return $status;
 359         }
 360
 361         /**
 362          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 363          */
 364         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 365                 try {
 366                         $cfOp->getLastResponse();
 367                 } catch ( BadContentTypeException $e ) {
 368                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 369                 } catch ( IOException $e ) {
 370                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 371                 }
 372         }
 373
 374         /**
 375          * @see FileBackendStore::doCopyInternal()
 376          * @return Status
 377          */
 378         protected function doCopyInternal( array $params ) {
 379                 $status = Status::newGood();
 380
 381                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 382                 if ( $srcRel === null ) {
 383                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 384                         return $status;
 385                 }
 386
 387                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 388                 if ( $dstRel === null ) {
 389                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 390                         return $status;
 391                 }
 392
 393                 // (a) Check the source/destination containers and destination object
 394                 try {
 395                         $sContObj = $this->getContainer( $srcCont );
 396                         $dContObj = $this->getContainer( $dstCont );
 397                         if ( empty( $params['overwrite'] ) &&
 398                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 399                         {
 400                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 401                                 return $status;
 402                         }
 403                 } catch ( NoSuchContainerException $e ) {
 404                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 405                         return $status;
 406                 } catch ( CloudFilesException $e ) { // some other exception?
 407                         $this->handleException( $e, $status, __METHOD__, $params );
 408                         return $status;
 409                 }
 410
 411                 // (b) Actually copy the file to the destination
 412                 try {
 413                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 414                         $hdrs = array(); // source file headers to override with new values
 415                         if ( isset( $params['disposition'] ) ) {
 416                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 417                         }
 418                         if ( !empty( $params['async'] ) ) { // deferred
 419                                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 420                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
 421                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 422                                         $status->value->affectedObjects[] = $dstObj;
 423                                 }
 424                         } else { // actually write the object in Swift
 425                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 426                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 427                                         $this->purgeCDNCache( array( $dstObj ) );
 428                                 }
 429                         }
 430                 } catch ( CDNNotEnabledException $e ) {
 431                         // CDN not enabled; nothing to see here
 432                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 433                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 434                 } catch ( CloudFilesException $e ) { // some other exception?
 435                         $this->handleException( $e, $status, __METHOD__, $params );
 436                 }
 437
 438                 return $status;
 439         }
 440
 441         /**
 442          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 443          */
 444         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 445                 try {
 446                         $cfOp->getLastResponse();
 447                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 448                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 449                 }
 450         }
 451
 452         /**
 453          * @see FileBackendStore::doMoveInternal()
 454          * @return Status
 455          */
 456         protected function doMoveInternal( array $params ) {
 457                 $status = Status::newGood();
 458
 459                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 460                 if ( $srcRel === null ) {
 461                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 462                         return $status;
 463                 }
 464
 465                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 466                 if ( $dstRel === null ) {
 467                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 468                         return $status;
 469                 }
 470
 471                 // (a) Check the source/destination containers and destination object
 472                 try {
 473                         $sContObj = $this->getContainer( $srcCont );
 474                         $dContObj = $this->getContainer( $dstCont );
 475                         if ( empty( $params['overwrite'] ) &&
 476                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 477                         {
 478                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 479                                 return $status;
 480                         }
 481                 } catch ( NoSuchContainerException $e ) {
 482                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 483                         return $status;
 484                 } catch ( CloudFilesException $e ) { // some other exception?
 485                         $this->handleException( $e, $status, __METHOD__, $params );
 486                         return $status;
 487                 }
 488
 489                 // (b) Actually move the file to the destination
 490                 try {
 491                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 492                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 493                         $hdrs = array(); // source file headers to override with new values
 494                         if ( isset( $params['disposition'] ) ) {
 495                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 496                         }
 497                         if ( !empty( $params['async'] ) ) { // deferred
 498                                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 499                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
 500                                 $status->value->affectedObjects[] = $srcObj;
 501                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 502                                         $status->value->affectedObjects[] = $dstObj;
 503                                 }
 504                         } else { // actually write the object in Swift
 505                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 506                                 $this->purgeCDNCache( array( $srcObj ) );
 507                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 508                                         $this->purgeCDNCache( array( $dstObj ) );
 509                                 }
 510                         }
 511                 } catch ( CDNNotEnabledException $e ) {
 512                         // CDN not enabled; nothing to see here
 513                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 514                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 515                 } catch ( CloudFilesException $e ) { // some other exception?
 516                         $this->handleException( $e, $status, __METHOD__, $params );
 517                 }
 518
 519                 return $status;
 520         }
 521
 522         /**
 523          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 524          */
 525         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 526                 try {
 527                         $cfOp->getLastResponse();
 528                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 529                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 530                 }
 531         }
 532
 533         /**
 534          * @see FileBackendStore::doDeleteInternal()
 535          * @return Status
 536          */
 537         protected function doDeleteInternal( array $params ) {
 538                 $status = Status::newGood();
 539
 540                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 541                 if ( $srcRel === null ) {
 542                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 543                         return $status;
 544                 }
 545
 546                 try {
 547                         $sContObj = $this->getContainer( $srcCont );
 548                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 549                         if ( !empty( $params['async'] ) ) { // deferred
 550                                 $op = $sContObj->delete_object_async( $srcRel );
 551                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
 552                                 $status->value->affectedObjects[] = $srcObj;
 553                         } else { // actually write the object in Swift
 554                                 $sContObj->delete_object( $srcRel );
 555                                 $this->purgeCDNCache( array( $srcObj ) );
 556                         }
 557                 } catch ( CDNNotEnabledException $e ) {
 558                         // CDN not enabled; nothing to see here
 559                 } catch ( NoSuchContainerException $e ) {
 560                         $status->fatal( 'backend-fail-delete', $params['src'] );
 561                 } catch ( NoSuchObjectException $e ) {
 562                         if ( empty( $params['ignoreMissingSource'] ) ) {
 563                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 564                         }
 565                 } catch ( CloudFilesException $e ) { // some other exception?
 566                         $this->handleException( $e, $status, __METHOD__, $params );
 567                 }
 568
 569                 return $status;
 570         }
 571
 572         /**
 573          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 574          */
 575         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 576                 try {
 577                         $cfOp->getLastResponse();
 578                 } catch ( NoSuchContainerException $e ) {
 579                         $status->fatal( 'backend-fail-delete', $params['src'] );
 580                 } catch ( NoSuchObjectException $e ) {
 581                         if ( empty( $params['ignoreMissingSource'] ) ) {
 582                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 583                         }
 584                 }
 585         }
 586
 587         /**
 588          * @see FileBackendStore::doPrepareInternal()
 589          * @return Status
 590          */
 591         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 592                 $status = Status::newGood();
 593
 594                 // (a) Check if container already exists
 595                 try {
 596                         $contObj = $this->getContainer( $fullCont );
 597                         // NoSuchContainerException not thrown: container must exist
 598                         return $status; // already exists
 599                 } catch ( NoSuchContainerException $e ) {
 600                         // NoSuchContainerException thrown: container does not exist
 601                 } catch ( CloudFilesException $e ) { // some other exception?
 602                         $this->handleException( $e, $status, __METHOD__, $params );
 603                         return $status;
 604                 }
 605
 606                 // (b) Create container as needed
 607                 try {
 608                         $contObj = $this->createContainer( $fullCont );
 609                         if ( !empty( $params['noAccess'] ) ) {
 610                                 // Make container private to end-users...
 611                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 612                         } else {
 613                                 // Make container public to end-users...
 614                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 615                         }
 616                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 617                                 $contObj->make_public( $this->swiftCDNExpiry );
 618                         }
 619                 } catch ( CDNNotEnabledException $e ) {
 620                         // CDN not enabled; nothing to see here
 621                 } catch ( CloudFilesException $e ) { // some other exception?
 622                         $this->handleException( $e, $status, __METHOD__, $params );
 623                         return $status;
 624                 }
 625
 626                 return $status;
 627         }
 628
 629         /**
 630          * @see FileBackendStore::doSecureInternal()
 631          * @return Status
 632          */
 633         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 634                 $status = Status::newGood();
 635                 if ( empty( $params['noAccess'] ) ) {
 636                         return $status; // nothing to do
 637                 }
 638
 639                 // Restrict container from end-users...
 640                 try {
 641                         // doPrepareInternal() should have been called,
 642                         // so the Swift container should already exist...
 643                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 644                         // NoSuchContainerException not thrown: container must exist
 645
 646                         // Make container private to end-users...
 647                         $status->merge( $this->setContainerAccess(
 648                                 $contObj,
 649                                 array( $this->auth->username ), // read
 650                                 array( $this->auth->username ) // write
 651                         ) );
 652                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 653                                 $contObj->make_private();
 654                         }
 655                 } catch ( CDNNotEnabledException $e ) {
 656                         // CDN not enabled; nothing to see here
 657                 } catch ( CloudFilesException $e ) { // some other exception?
 658                         $this->handleException( $e, $status, __METHOD__, $params );
 659                 }
 660
 661                 return $status;
 662         }
 663
 664         /**
 665          * @see FileBackendStore::doPublishInternal()
 666          * @return Status
 667          */
 668         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 669                 $status = Status::newGood();
 670
 671                 // Unrestrict container from end-users...
 672                 try {
 673                         // doPrepareInternal() should have been called,
 674                         // so the Swift container should already exist...
 675                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 676                         // NoSuchContainerException not thrown: container must exist
 677
 678                         // Make container public to end-users...
 679                         if ( $this->swiftAnonUser != '' ) {
 680                                 $status->merge( $this->setContainerAccess(
 681                                         $contObj,
 682                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 683                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 684                                 ) );
 685                         } else {
 686                                 $status->merge( $this->setContainerAccess(
 687                                         $contObj,
 688                                         array( $this->auth->username, '.r:*' ), // read
 689                                         array( $this->auth->username ) // write
 690                                 ) );
 691                         }
 692                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 693                                 $contObj->make_public();
 694                         }
 695                 } catch ( CDNNotEnabledException $e ) {
 696                         // CDN not enabled; nothing to see here
 697                 } catch ( CloudFilesException $e ) { // some other exception?
 698                         $this->handleException( $e, $status, __METHOD__, $params );
 699                 }
 700
 701                 return $status;
 702         }
 703
 704         /**
 705          * @see FileBackendStore::doCleanInternal()
 706          * @return Status
 707          */
 708         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 709                 $status = Status::newGood();
 710
 711                 // Only containers themselves can be removed, all else is virtual
 712                 if ( $dir != '' ) {
 713                         return $status; // nothing to do
 714                 }
 715
 716                 // (a) Check the container
 717                 try {
 718                         $contObj = $this->getContainer( $fullCont, true );
 719                 } catch ( NoSuchContainerException $e ) {
 720                         return $status; // ok, nothing to do
 721                 } catch ( CloudFilesException $e ) { // some other exception?
 722                         $this->handleException( $e, $status, __METHOD__, $params );
 723                         return $status;
 724                 }
 725
 726                 // (b) Delete the container if empty
 727                 if ( $contObj->object_count == 0 ) {
 728                         try {
 729                                 $this->deleteContainer( $fullCont );
 730                         } catch ( NoSuchContainerException $e ) {
 731                                 return $status; // race?
 732                         } catch ( NonEmptyContainerException $e ) {
 733                                 return $status; // race? consistency delay?
 734                         } catch ( CloudFilesException $e ) { // some other exception?
 735                                 $this->handleException( $e, $status, __METHOD__, $params );
 736                                 return $status;
 737                         }
 738                 }
 739
 740                 return $status;
 741         }
 742
 743         /**
 744          * @see FileBackendStore::doFileExists()
 745          * @return array|bool|null
 746          */
 747         protected function doGetFileStat( array $params ) {
 748                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 749                 if ( $srcRel === null ) {
 750                         return false; // invalid storage path
 751                 }
 752
 753                 $stat = false;
 754                 try {
 755                         $contObj = $this->getContainer( $srcCont );
 756                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 757                         $this->addMissingMetadata( $srcObj, $params['src'] );
 758                         $stat = array(
 759                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 760                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 761                                 'size'  => (int)$srcObj->content_length,
 762                                 'sha1'  => $srcObj->getMetadataValue( 'Sha1base36' )
 763                         );
 764                 } catch ( NoSuchContainerException $e ) {
 765                 } catch ( NoSuchObjectException $e ) {
 766                 } catch ( CloudFilesException $e ) { // some other exception?
 767                         $stat = null;
 768                         $this->handleException( $e, null, __METHOD__, $params );
 769                 }
 770
 771                 return $stat;
 772         }
 773
 774         /**
 775          * Fill in any missing object metadata and save it to Swift
 776          *
 777          * @param $obj CF_Object
 778          * @param $path string Storage path to object
 779          * @return bool Success
 780          * @throws Exception cloudfiles exceptions
 781          */
 782         protected function addMissingMetadata( CF_Object $obj, $path ) {
 783                 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
 784                         return true; // nothing to do
 785                 }
 786                 wfProfileIn( __METHOD__ );
 787                 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
 788                 $status = Status::newGood();
 789                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 790                 if ( $status->isOK() ) {
 791                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 792                         if ( $tmpFile ) {
 793                                 $hash = $tmpFile->getSha1Base36();
 794                                 if ( $hash !== false ) {
 795                                         $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
 796                                         $obj->sync_metadata(); // save to Swift
 797                                         wfProfileOut( __METHOD__ );
 798                                         return true; // success
 799                                 }
 800                         }
 801                 }
 802                 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
 803                 wfProfileOut( __METHOD__ );
 804                 return false; // failed
 805         }
 806
 807         /**
 808          * @see FileBackendStore::doGetFileContentsMulti()
 809          * @return Array
 810          */
 811         protected function doGetFileContentsMulti( array $params ) {
 812                 $contents = array();
 813
 814                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
 815                 // Blindly create tmp files and stream to them, catching any exception if the file does
 816                 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
 817                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
 818                         $cfOps = array(); // (path => CF_Async_Op)
 819
 820                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
 821                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
 822                                 if ( $srcRel === null ) {
 823                                         $contents[$path] = false;
 824                                         continue;
 825                                 }
 826                                 $data = false;
 827                                 try {
 828                                         $sContObj = $this->getContainer( $srcCont );
 829                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 830                                         // Get source file extension
 831                                         $ext = FileBackend::extensionFromPath( $path );
 832                                         // Create a new temporary memory file...
 833                                         $handle = fopen( 'php://temp', 'wb' );
 834                                         if ( $handle ) {
 835                                                 $headers = $this->headersFromParams( $params );
 836                                                 if ( count( $pathBatch ) > 1 ) {
 837                                                         $cfOps[$path] = $obj->stream_async( $handle, $headers );
 838                                                         $cfOps[$path]->_file_handle = $handle; // close this later
 839                                                 } else {
 840                                                         $obj->stream( $handle, $headers );
 841                                                         rewind( $handle ); // start from the beginning
 842                                                         $data = stream_get_contents( $handle );
 843                                                         fclose( $handle );
 844                                                 }
 845                                         } else {
 846                                                 $data = false;
 847                                         }
 848                                 } catch ( NoSuchContainerException $e ) {
 849                                         $data = false;
 850                                 } catch ( NoSuchObjectException $e ) {
 851                                         $data = false;
 852                                 } catch ( CloudFilesException $e ) { // some other exception?
 853                                         $data = false;
 854                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 855                                 }
 856                                 $contents[$path] = $data;
 857                         }
 858
 859                         $batch = new CF_Async_Op_Batch( $cfOps );
 860                         $cfOps = $batch->execute();
 861                         foreach ( $cfOps as $path => $cfOp ) {
 862                                 try {
 863                                         $cfOp->getLastResponse();
 864                                         rewind( $cfOp->_file_handle ); // start from the beginning
 865                                         $contents[$path] = stream_get_contents( $cfOp->_file_handle );
 866                                 } catch ( NoSuchContainerException $e ) {
 867                                         $contents[$path] = false;
 868                                 } catch ( NoSuchObjectException $e ) {
 869                                         $contents[$path] = false;
 870                                 } catch ( CloudFilesException $e ) { // some other exception?
 871                                         $contents[$path] = false;
 872                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 873                                 }
 874                                 fclose( $cfOp->_file_handle ); // close open handle
 875                         }
 876                 }
 877
 878                 return $contents;
 879         }
 880
 881         /**
 882          * @see FileBackendStore::doDirectoryExists()
 883          * @return bool|null
 884          */
 885         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 886                 try {
 887                         $container = $this->getContainer( $fullCont );
 888                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 889                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 890                 } catch ( NoSuchContainerException $e ) {
 891                         return false;
 892                 } catch ( CloudFilesException $e ) { // some other exception?
 893                         $this->handleException( $e, null, __METHOD__,
 894                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 895                 }
 896
 897                 return null; // error
 898         }
 899
 900         /**
 901          * @see FileBackendStore::getDirectoryListInternal()
 902          * @return SwiftFileBackendDirList
 903          */
 904         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 905                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 906         }
 907
 908         /**
 909          * @see FileBackendStore::getFileListInternal()
 910          * @return SwiftFileBackendFileList
 911          */
 912         public function getFileListInternal( $fullCont, $dir, array $params ) {
 913                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 914         }
 915
 916         /**
 917          * Do not call this function outside of SwiftFileBackendFileList
 918          *
 919          * @param $fullCont string Resolved container name
 920          * @param $dir string Resolved storage directory with no trailing slash
 921          * @param $after string|null Storage path of file to list items after
 922          * @param $limit integer Max number of items to list
 923          * @param $params Array Includes flag for 'topOnly'
 924          * @return Array List of relative paths of dirs directly under $dir
 925          */
 926         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 927                 $dirs = array();
 928                 if ( $after === INF ) {
 929                         return $dirs; // nothing more
 930                 }
 931                 wfProfileIn( __METHOD__ . '-' . $this->name );
 932
 933                 try {
 934                         $container = $this->getContainer( $fullCont );
 935                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 936                         // Non-recursive: only list dirs right under $dir
 937                         if ( !empty( $params['topOnly'] ) ) {
 938                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 939                                 foreach ( $objects as $object ) { // files and dirs
 940                                         if ( substr( $object, -1 ) === '/' ) {
 941                                                 $dirs[] = $object; // directories end in '/'
 942                                         }
 943                                 }
 944                         // Recursive: list all dirs under $dir and its subdirs
 945                         } else {
 946                                 // Get directory from last item of prior page
 947                                 $lastDir = $this->getParentDir( $after ); // must be first page
 948                                 $objects = $container->list_objects( $limit, $after, $prefix );
 949                                 foreach ( $objects as $object ) { // files
 950                                         $objectDir = $this->getParentDir( $object ); // directory of object
 951                                         if ( $objectDir !== false ) { // file has a parent dir
 952                                                 // Swift stores paths in UTF-8, using binary sorting.
 953                                                 // See function "create_container_table" in common/db.py.
 954                                                 // If a directory is not "greater" than the last one,
 955                                                 // then it was already listed by the calling iterator.
 956                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
 957                                                         $pDir = $objectDir;
 958                                                         do { // add dir and all its parent dirs
 959                                                                 $dirs[] = "{$pDir}/";
 960                                                                 $pDir = $this->getParentDir( $pDir );
 961                                                         } while ( $pDir !== false // sanity
 962                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
 963                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 964                                                         );
 965                                                 }
 966                                                 $lastDir = $objectDir;
 967                                         }
 968                                 }
 969                         }
 970                         if ( count( $objects ) < $limit ) {
 971                                 $after = INF; // avoid a second RTT
 972                         } else {
 973                                 $after = end( $objects ); // update last item
 974                         }
 975                 } catch ( NoSuchContainerException $e ) {
 976                 } catch ( CloudFilesException $e ) { // some other exception?
 977                         $this->handleException( $e, null, __METHOD__,
 978                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 979                 }
 980
 981                 wfProfileOut( __METHOD__ . '-' . $this->name );
 982                 return $dirs;
 983         }
 984
 985         protected function getParentDir( $path ) {
 986                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 987         }
 988
 989         /**
 990          * Do not call this function outside of SwiftFileBackendFileList
 991          *
 992          * @param $fullCont string Resolved container name
 993          * @param $dir string Resolved storage directory with no trailing slash
 994          * @param $after string|null Storage path of file to list items after
 995          * @param $limit integer Max number of items to list
 996          * @param $params Array Includes flag for 'topOnly'
 997          * @return Array List of relative paths of files under $dir
 998          */
 999         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1000                 $files = array();
1001                 if ( $after === INF ) {
1002                         return $files; // nothing more
1003                 }
1004                 wfProfileIn( __METHOD__ . '-' . $this->name );
1005
1006                 try {
1007                         $container = $this->getContainer( $fullCont );
1008                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
1009                         // Non-recursive: only list files right under $dir
1010                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
1011                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
1012                                 foreach ( $objects as $object ) {
1013                                         if ( substr( $object, -1 ) !== '/' ) {
1014                                                 $files[] = $object; // directories end in '/'
1015                                         }
1016                                 }
1017                         // Recursive: list all files under $dir and its subdirs
1018                         } else { // files
1019                                 $objects = $container->list_objects( $limit, $after, $prefix );
1020                                 $files = $objects;
1021                         }
1022                         if ( count( $objects ) < $limit ) {
1023                                 $after = INF; // avoid a second RTT
1024                         } else {
1025                                 $after = end( $objects ); // update last item
1026                         }
1027                 } catch ( NoSuchContainerException $e ) {
1028                 } catch ( CloudFilesException $e ) { // some other exception?
1029                         $this->handleException( $e, null, __METHOD__,
1030                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1031                 }
1032
1033                 wfProfileOut( __METHOD__ . '-' . $this->name );
1034                 return $files;
1035         }
1036
1037         /**
1038          * @see FileBackendStore::doGetFileSha1base36()
1039          * @return bool
1040          */
1041         protected function doGetFileSha1base36( array $params ) {
1042                 $stat = $this->getFileStat( $params );
1043                 if ( $stat ) {
1044                         return $stat['sha1'];
1045                 } else {
1046                         return false;
1047                 }
1048         }
1049
1050         /**
1051          * @see FileBackendStore::doStreamFile()
1052          * @return Status
1053          */
1054         protected function doStreamFile( array $params ) {
1055                 $status = Status::newGood();
1056
1057                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1058                 if ( $srcRel === null ) {
1059                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1060                 }
1061
1062                 try {
1063                         $cont = $this->getContainer( $srcCont );
1064                 } catch ( NoSuchContainerException $e ) {
1065                         $status->fatal( 'backend-fail-stream', $params['src'] );
1066                         return $status;
1067                 } catch ( CloudFilesException $e ) { // some other exception?
1068                         $this->handleException( $e, $status, __METHOD__, $params );
1069                         return $status;
1070                 }
1071
1072                 try {
1073                         $output = fopen( 'php://output', 'wb' );
1074                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1075                         $obj->stream( $output, $this->headersFromParams( $params ) );
1076                 } catch ( NoSuchObjectException $e ) {
1077                         $status->fatal( 'backend-fail-stream', $params['src'] );
1078                 } catch ( CloudFilesException $e ) { // some other exception?
1079                         $this->handleException( $e, $status, __METHOD__, $params );
1080                 }
1081
1082                 return $status;
1083         }
1084
1085         /**
1086          * @see FileBackendStore::doGetLocalCopyMulti()
1087          * @return null|TempFSFile
1088          */
1089         protected function doGetLocalCopyMulti( array $params ) {
1090                 $tmpFiles = array();
1091
1092                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1093                 // Blindly create tmp files and stream to them, catching any exception if the file does
1094                 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1095                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1096                         $cfOps = array(); // (path => CF_Async_Op)
1097
1098                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1099                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1100                                 if ( $srcRel === null ) {
1101                                         $tmpFiles[$path] = null;
1102                                         continue;
1103                                 }
1104                                 $tmpFile = null;
1105                                 try {
1106                                         $sContObj = $this->getContainer( $srcCont );
1107                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1108                                         // Get source file extension
1109                                         $ext = FileBackend::extensionFromPath( $path );
1110                                         // Create a new temporary file...
1111                                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1112                                         if ( $tmpFile ) {
1113                                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1114                                                 if ( $handle ) {
1115                                                         $headers = $this->headersFromParams( $params );
1116                                                         if ( count( $pathBatch ) > 1 ) {
1117                                                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1118                                                                 $cfOps[$path]->_file_handle = $handle; // close this later
1119                                                         } else {
1120                                                                 $obj->stream( $handle, $headers );
1121                                                                 fclose( $handle );
1122                                                         }
1123                                                 } else {
1124                                                         $tmpFile = null;
1125                                                 }
1126                                         }
1127                                 } catch ( NoSuchContainerException $e ) {
1128                                         $tmpFile = null;
1129                                 } catch ( NoSuchObjectException $e ) {
1130                                         $tmpFile = null;
1131                                 } catch ( CloudFilesException $e ) { // some other exception?
1132                                         $tmpFile = null;
1133                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1134                                 }
1135                                 $tmpFiles[$path] = $tmpFile;
1136                         }
1137
1138                         $batch = new CF_Async_Op_Batch( $cfOps );
1139                         $cfOps = $batch->execute();
1140                         foreach ( $cfOps as $path => $cfOp ) {
1141                                 try {
1142                                         $cfOp->getLastResponse();
1143                                 } catch ( NoSuchContainerException $e ) {
1144                                         $tmpFiles[$path] = null;
1145                                 } catch ( NoSuchObjectException $e ) {
1146                                         $tmpFiles[$path] = null;
1147                                 } catch ( CloudFilesException $e ) { // some other exception?
1148                                         $tmpFiles[$path] = null;
1149                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1150                                 }
1151                                 fclose( $cfOp->_file_handle ); // close open handle
1152                         }
1153                 }
1154
1155                 return $tmpFiles;
1156         }
1157
1158         /**
1159          * @see FileBackendStore::directoriesAreVirtual()
1160          * @return bool
1161          */
1162         protected function directoriesAreVirtual() {
1163                 return true;
1164         }
1165
1166         /**
1167          * Get headers to send to Swift when reading a file based
1168          * on a FileBackend params array, e.g. that of getLocalCopy().
1169          * $params is currently only checked for a 'latest' flag.
1170          *
1171          * @param $params Array
1172          * @return Array
1173          */
1174         protected function headersFromParams( array $params ) {
1175                 $hdrs = array();
1176                 if ( !empty( $params['latest'] ) ) {
1177                         $hdrs[] = 'X-Newest: true';
1178                 }
1179                 return $hdrs;
1180         }
1181
1182         /**
1183          * @see FileBackendStore::doExecuteOpHandlesInternal()
1184          * @return Array List of corresponding Status objects
1185          */
1186         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1187                 $statuses = array();
1188
1189                 $cfOps = array(); // list of CF_Async_Op objects
1190                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1191                         $cfOps[$index] = $fileOpHandle->cfOp;
1192                 }
1193                 $batch = new CF_Async_Op_Batch( $cfOps );
1194
1195                 $cfOps = $batch->execute();
1196                 foreach ( $cfOps as $index => $cfOp ) {
1197                         $status = Status::newGood();
1198                         try { // catch exceptions; update status
1199                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1200                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1201                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1202                         } catch ( CloudFilesException $e ) { // some other exception?
1203                                 $this->handleException( $e, $status,
1204                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1205                         }
1206                         $statuses[$index] = $status;
1207                 }
1208
1209                 return $statuses;
1210         }
1211
1212         /**
1213          * Set read/write permissions for a Swift container.
1214          *
1215          * $readGrps is a list of the possible criteria for a request to have
1216          * access to read a container. Each item is one of the following formats:
1217          *   - account:user       : Grants access if the request is by the given user
1218          *   - ".r:<regex>"       : Grants access if the request is from a referrer host that
1219          *                          matches the expression and the request is not for a listing.
1220          *                          Setting this to '*' effectively makes a container public.
1221          *   -".rlistings:<regex>": Grants access if the request is from a referrer host that
1222          *                          matches the expression and the request for a listing.
1223          *
1224          * $writeGrps is a list of the possible criteria for a request to have
1225          * access to write to a container. Each item is of the following format:
1226          *   - account:user       : Grants access if the request is by the given user
1227          *
1228          * @see http://swift.openstack.org/misc.html#acls
1229          *
1230          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1231          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1232          *
1233          * @param $contObj CF_Container Swift container
1234          * @param $readGrps Array List of read access routes
1235          * @param $writeGrps Array List of write access routes
1236          * @return Status
1237          */
1238         protected function setContainerAccess(
1239                 CF_Container $contObj, array $readGrps, array $writeGrps
1240         ) {
1241                 $creds = $contObj->cfs_auth->export_credentials();
1242
1243                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1244
1245                 // Note: 10 second timeout consistent with php-cloudfiles
1246                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1247                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1248                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1249                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1250
1251                 return $req->execute(); // should return 204
1252         }
1253
1254         /**
1255          * Purge the CDN cache of affected objects if CDN caching is enabled.
1256          * This is for Rackspace/Akamai CDNs.
1257          *
1258          * @param $objects Array List of CF_Object items
1259          * @return void
1260          */
1261         public function purgeCDNCache( array $objects ) {
1262                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1263                         foreach ( $objects as $object ) {
1264                                 try {
1265                                         $object->purge_from_cdn();
1266                                 } catch ( CDNNotEnabledException $e ) {
1267                                         // CDN not enabled; nothing to see here
1268                                 } catch ( CloudFilesException $e ) {
1269                                         $this->handleException( $e, null, __METHOD__,
1270                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1271                                 }
1272                         }
1273                 }
1274         }
1275
1276         /**
1277          * Get an authenticated connection handle to the Swift proxy
1278          *
1279          * @return CF_Connection|bool False on failure
1280          * @throws CloudFilesException
1281          */
1282         protected function getConnection() {
1283                 if ( $this->connException instanceof CloudFilesException ) {
1284                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1285                                 throw $this->connException; // failed last attempt; don't bother
1286                         } else { // actually retry this time
1287                                 $this->connException = null;
1288                                 $this->connErrorTime = 0;
1289                         }
1290                 }
1291                 // Session keys expire after a while, so we renew them periodically
1292                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1293                 // Authenticate with proxy and get a session key...
1294                 if ( !$this->conn || $reAuth ) {
1295                         $this->sessionStarted = 0;
1296                         $this->connContainerCache->clear();
1297                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1298                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1299                         if ( is_array( $creds ) ) { // cache hit
1300                                 $this->auth->load_cached_credentials(
1301                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1302                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1303                         } else { // cache miss
1304                                 try {
1305                                         $this->auth->authenticate();
1306                                         $creds = $this->auth->export_credentials();
1307                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1308                                         $this->sessionStarted = time();
1309                                 } catch ( CloudFilesException $e ) {
1310                                         $this->connException = $e; // don't keep re-trying
1311                                         $this->connErrorTime = time();
1312                                         throw $e; // throw it back
1313                                 }
1314                         }
1315                         if ( $this->conn ) { // re-authorizing?
1316                                 $this->conn->close(); // close active cURL handles in CF_Http object
1317                         }
1318                         $this->conn = new CF_Connection( $this->auth );
1319                 }
1320                 return $this->conn;
1321         }
1322
1323         /**
1324          * Close the connection to the Swift proxy
1325          *
1326          * @return void
1327          */
1328         protected function closeConnection() {
1329                 if ( $this->conn ) {
1330                         $this->conn->close(); // close active cURL handles in CF_Http object
1331                         $this->sessionStarted = 0;
1332                         $this->connContainerCache->clear();
1333                 }
1334         }
1335
1336         /**
1337          * Get the cache key for a container
1338          *
1339          * @param $username string
1340          * @return string
1341          */
1342         private function getCredsCacheKey( $username ) {
1343                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1344         }
1345
1346         /**
1347          * Get a Swift container object, possibly from process cache.
1348          * Use $reCache if the file count or byte count is needed.
1349          *
1350          * @param $container string Container name
1351          * @param $bypassCache bool Bypass all caches and load from Swift
1352          * @return CF_Container
1353          * @throws CloudFilesException
1354          */
1355         protected function getContainer( $container, $bypassCache = false ) {
1356                 $conn = $this->getConnection(); // Swift proxy connection
1357                 if ( $bypassCache ) { // purge cache
1358                         $this->connContainerCache->clear( $container );
1359                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1360                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1361                 }
1362                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1363                         $contObj = $conn->get_container( $container );
1364                         // NoSuchContainerException not thrown: container must exist
1365                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1366                         if ( !$bypassCache ) {
1367                                 $this->setContainerCache( $container, // update persistent cache
1368                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1369                                 );
1370                         }
1371                 }
1372                 return $this->connContainerCache->get( $container, 'obj' );
1373         }
1374
1375         /**
1376          * Create a Swift container
1377          *
1378          * @param $container string Container name
1379          * @return CF_Container
1380          * @throws CloudFilesException
1381          */
1382         protected function createContainer( $container ) {
1383                 $conn = $this->getConnection(); // Swift proxy connection
1384                 $contObj = $conn->create_container( $container );
1385                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1386                 return $contObj;
1387         }
1388
1389         /**
1390          * Delete a Swift container
1391          *
1392          * @param $container string Container name
1393          * @return void
1394          * @throws CloudFilesException
1395          */
1396         protected function deleteContainer( $container ) {
1397                 $conn = $this->getConnection(); // Swift proxy connection
1398                 $this->connContainerCache->clear( $container ); // purge
1399                 $conn->delete_container( $container );
1400         }
1401
1402         /**
1403          * @see FileBackendStore::doPrimeContainerCache()
1404          * @return void
1405          */
1406         protected function doPrimeContainerCache( array $containerInfo ) {
1407                 try {
1408                         $conn = $this->getConnection(); // Swift proxy connection
1409                         foreach ( $containerInfo as $container => $info ) {
1410                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1411                                         $container, $info['count'], $info['bytes'] );
1412                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1413                         }
1414                 } catch ( CloudFilesException $e ) { // some other exception?
1415                         $this->handleException( $e, null, __METHOD__, array() );
1416                 }
1417         }
1418
1419         /**
1420          * Log an unexpected exception for this backend.
1421          * This also sets the Status object to have a fatal error.
1422          *
1423          * @param $e Exception
1424          * @param $status Status|null
1425          * @param $func string
1426          * @param $params Array
1427          * @return void
1428          */
1429         protected function handleException( Exception $e, $status, $func, array $params ) {
1430                 if ( $status instanceof Status ) {
1431                         if ( $e instanceof AuthenticationException ) {
1432                                 $status->fatal( 'backend-fail-connect', $this->name );
1433                         } else {
1434                                 $status->fatal( 'backend-fail-internal', $this->name );
1435                         }
1436                 }
1437                 if ( $e->getMessage() ) {
1438                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1439                 }
1440                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1441                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1442                         $this->closeConnection(); // force a re-connect and re-auth next time
1443                 }
1444                 wfDebugLog( 'SwiftBackend',
1445                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1446                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1447                 );
1448         }
1449 }
1450
1451 /**
1452  * @see FileBackendStoreOpHandle
1453  */
1454 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1455         /** @var CF_Async_Op */
1456         public $cfOp;
1457         /** @var Array */
1458         public $affectedObjects = array();
1459
1460         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1461                 $this->backend = $backend;
1462                 $this->params = $params;
1463                 $this->call = $call;
1464                 $this->cfOp = $cfOp;
1465         }
1466 }
1467
1468 /**
1469  * SwiftFileBackend helper class to page through listings.
1470  * Swift also has a listing limit of 10,000 objects for sanity.
1471  * Do not use this class from places outside SwiftFileBackend.
1472  *
1473  * @ingroup FileBackend
1474  */
1475 abstract class SwiftFileBackendList implements Iterator {
1476         /** @var Array */
1477         protected $bufferIter = array();
1478         protected $bufferAfter = null; // string; list items *after* this path
1479         protected $pos = 0; // integer
1480         /** @var Array */
1481         protected $params = array();
1482
1483         /** @var SwiftFileBackend */
1484         protected $backend;
1485         protected $container; // string; container name
1486         protected $dir; // string; storage directory
1487         protected $suffixStart; // integer
1488
1489         const PAGE_SIZE = 9000; // file listing buffer size
1490
1491         /**
1492          * @param $backend SwiftFileBackend
1493          * @param $fullCont string Resolved container name
1494          * @param $dir string Resolved directory relative to container
1495          * @param $params Array
1496          */
1497         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1498                 $this->backend = $backend;
1499                 $this->container = $fullCont;
1500                 $this->dir = $dir;
1501                 if ( substr( $this->dir, -1 ) === '/' ) {
1502                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1503                 }
1504                 if ( $this->dir == '' ) { // whole container
1505                         $this->suffixStart = 0;
1506                 } else { // dir within container
1507                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1508                 }
1509                 $this->params = $params;
1510         }
1511
1512         /**
1513          * @see Iterator::key()
1514          * @return integer
1515          */
1516         public function key() {
1517                 return $this->pos;
1518         }
1519
1520         /**
1521          * @see Iterator::next()
1522          * @return void
1523          */
1524         public function next() {
1525                 // Advance to the next file in the page
1526                 next( $this->bufferIter );
1527                 ++$this->pos;
1528                 // Check if there are no files left in this page and
1529                 // advance to the next page if this page was not empty.
1530                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1531                         $this->bufferIter = $this->pageFromList(
1532                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1533                         ); // updates $this->bufferAfter
1534                 }
1535         }
1536
1537         /**
1538          * @see Iterator::rewind()
1539          * @return void
1540          */
1541         public function rewind() {
1542                 $this->pos = 0;
1543                 $this->bufferAfter = null;
1544                 $this->bufferIter = $this->pageFromList(
1545                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1546                 ); // updates $this->bufferAfter
1547         }
1548
1549         /**
1550          * @see Iterator::valid()
1551          * @return bool
1552          */
1553         public function valid() {
1554                 if ( $this->bufferIter === null ) {
1555                         return false; // some failure?
1556                 } else {
1557                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1558                 }
1559         }
1560
1561         /**
1562          * Get the given list portion (page)
1563          *
1564          * @param $container string Resolved container name
1565          * @param $dir string Resolved path relative to container
1566          * @param $after string|null
1567          * @param $limit integer
1568          * @param $params Array
1569          * @return Traversable|Array|null Returns null on failure
1570          */
1571         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1572 }
1573
1574 /**
1575  * Iterator for listing directories
1576  */
1577 class SwiftFileBackendDirList extends SwiftFileBackendList {
1578         /**
1579          * @see Iterator::current()
1580          * @return string|bool String (relative path) or false
1581          */
1582         public function current() {
1583                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1584         }
1585
1586         /**
1587          * @see SwiftFileBackendList::pageFromList()
1588          * @return Array|null
1589          */
1590         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1591                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1592         }
1593 }
1594
1595 /**
1596  * Iterator for listing regular files
1597  */
1598 class SwiftFileBackendFileList extends SwiftFileBackendList {
1599         /**
1600          * @see Iterator::current()
1601          * @return string|bool String (relative path) or false
1602          */
1603         public function current() {
1604                 return substr( current( $this->bufferIter ), $this->suffixStart );
1605         }
1606
1607         /**
1608          * @see SwiftFileBackendList::pageFromList()
1609          * @return Array|null
1610          */
1611         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1612                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1613         }
1614 }